#!/usr/bin/perl

# This script is run from a central monitoring point (rose.cs.berkeley.edu)
# to check if i3 servers are running on the specified PlanetLab nodes.
# It updates a web page and status text file containing the current status
# of the different i3 servers.  The web page and the text file are created
# in the local directory; they must be copied to the public www directory by
# another script which periodically invokes this script.

use strict 'vars';
package main;

use Getopt::Std;
use Cwd;

my $ssh_options = "-o \"ConnectTimeout=20\" -o \"KeepAlive=yes\" -o \"PreferredAuthentications=publickey\" -n";

# The user name for logging into the PlanetLab nodes
my $user_name = "ucb_i3";

# The name of the i3 server binary file
my $i3_server_binary = "i3serverd";

use vars qw($opt_f $opt_o $opt_l);

#get command line options
getopts('f:o:l:');
if ($opt_f eq "" || $opt_o eq "" || $opt_l eq "") {
    die "Usage: update_i3_servers_web_status.pl -f <conf_file> -o <output_prefix> -l <lat-long file>\n";
}

my %ip2name = (), my %name2ip = ();
&get_host_map;

my %lat_long = ();
&get_lat_long;

open (HTML, ">$opt_o.html");
open (TXT, ">$opt_o.txt");
open (CRON_STATUS, ">$opt_o.cron_status");

my $restart_file_name = "toRestart.txt";
open (RESTART_FILE, ">$restart_file_name");

open (CONF_FILE, $opt_f) || die "Cannot open conf file\n";

my %hosts = ();
my $line, my @arr, my $cmd;
my %i3_count = (), my %init_count = ();
my %tcp_count = (), my %id = (); my %port = ();
my %uptime_hash = ();
my %cron_count = ();

my $total_count  = 0;   # total number of i3 servers in the list
my $total_alive_count = 0; # total number of i3 servers that are alive
my $threshold_percent = 50; # if the number of  i3 servers running falls below this percentage,send email alert.
my @email_alert_list = ("dilip.antony.joseph\@gmail.com", "dilip\@cs.berkeley.edu", "istoica\@cs.berkeley.edu");

# Should we try to restart i3 server monitors on which they are currently not running
my $attempt_restart_monitors_flag = 0; #0 --> no, 1--> yes

# If the number of i3 servers running is below the threshold, an email alert will be sent.
# However emails are sent no faster than one every min_interval_between_mails minutes.
my $min_interval_between_warn_mails = 3 * 60; #time in minutes.

my $min_interval_between_status_mails = 60; #time in minutes.

my $count = 0;
my $restart_monitors_flag = 0;

while ($line = <CONF_FILE>) {

    $total_count ++;
    
    chomp $line;
    @arr = split(/ /, $line);
    my $host = $arr[0]; my $port = $arr[1]; my $chord_port = $arr[2];
    $hosts{$arr[0]} = $host;
    $port{$arr[0]} = $port;
    $id{$arr[0]} = $arr[3];
    
    print "$count:->IP: $arr[0], Port: $arr[1]\n";
    $count ++;
		
    # check i3 server and chord
    print "Checking i3 server and chord\n";
    
    my $tmp_file = "/tmp/i3_periodic_tmpfile";
    system ("rm $tmp_file");

my %lat_long = ();
&get_lat_long;
    system("ssh $ssh_options -o \"StrictHostKeyChecking no\" $user_name\@$host \"ps ax\">$tmp_file");
    open PS, "$tmp_file";
    my $got_uptime = 0;

my $tmpPScount = 0;
    while (<PS>) {
	   
	print "In PS $tmpPScount: $_\n";
	$tmpPScount++;
	 
	    if ($_ =~ /$i3_server_binary/) {
	        $i3_count{$host}++;
	    } elsif ($_ =~ /initiator/ && $_ =~ /$chord_port/) {
	        $init_count{$host}++;
	    } elsif ($_ =~ /tcp_receiver/ && $_ =~ /$port/) {
	        $tcp_count{$host}++;
	    } elsif ($_ =~ /crond/) {
            	$cron_count{$host}++;
            }
    }
    close PS;

	
    print "Getting uptime\n";	
	    if ($got_uptime == 0) {
	        my $i; my $status = 0;
	        open UP, "ssh $ssh_options -o \"StrictHostKeyChecking no\" $user_name\@$host \"uptime\"|";
	        my $uptime = <UP>;
	        my @spl_up_arr = split(/,/, $uptime);
	        my @up_spl_up_arr = split(/up/, $spl_up_arr[0]);
	        $uptime_hash{$host} = "$up_spl_up_arr[1]";
	        
		print "Starting to parse uptime\n";
            	for ($i = 1; $i < int(scalar(@spl_up_arr)); $i++) {
		        if ($spl_up_arr[$i] =~ /users/) {
		            last;
		        }
		        $uptime_hash{$host} = $uptime_hash{$host}."$spl_up_arr[$i]";
			print "Parsing uptime\n";
	    	}
	       close UP;
	    }
	    
        $got_uptime = 1;

    
    # Check if the /etc/crontab file contains entries for periodic checking of i3 server status
    print "Checking if /etc/crontab exists\n";
    open CRONTAB, "ssh $ssh_options -o \"StrictHostKeyChecking no\" $user_name\@$host \"cat /etc/crontab\"|";
    my $crontab_flag = 0;
    while (<CRONTAB>) {
	if ( $_ =~ /i3server_check.pl/) {
		$crontab_flag = 1;
	}
    }

    # Check if /etc/rc.local starts crond on boot up.
    print "Checking if /etc/rc.local starts crond on boot\n";
    my $crond_rclocal_flag = 0;
    open RCLOCAL, "ssh $ssh_options -o \"StrictHostKeyChecking no\" $user_name\@$host \"cat /etc/rc.local\"|";
    open RCLOCAL, "cat /etc/rc.local |";
    while (<RCLOCAL>) {
	if ( $_ =~ /crond/) {
		$crond_rclocal_flag = 1;
	}
    }

    if ($cron_count{$host} < 1 || $crontab_flag == 0 || $crond_rclocal_flag == 0) {
            $restart_monitors_flag++;
            print RESTART_FILE "$host\n";     
    }

    if ($i3_count{$host} == 2) {
        $total_alive_count ++;
    }
}

close RESTART_FILE;

my $percent_alive = ($total_alive_count /$total_count) * 100.00;

my $sec, my $min, my $hr,my  $dy,my  $mnth,my  $yr,my  $wkDay,my  $dayYear,my  $is_dst;
($sec, $min, $hr, $dy, $mnth, $yr, $wkDay, $dayYear, $is_dst) = localtime(time);
   
if ($hr == 7 || $hr == 13) {
    my $status_mail_last_sent_file = "status_mail_last_sent.txt";
    open (LAST_SENT_FILE, "$status_mail_last_sent_file");

    my $status_mail_last_sent_time = <LAST_SENT_FILE>;

    close (LAST_SENT_FILE);
    
    my $currtime = time();
   
    if ($currtime - $status_mail_last_sent_time > $min_interval_between_status_mails * 60 ) {
        open (LAST_SENT_FILE, ">$status_mail_last_sent_file");
        print LAST_SENT_FILE $currtime;
        close LAST_SENT_FILE;

        open (MAIL_FILE, ">mailfile.tmp");
        print MAIL_FILE $percent_alive." % of the i3 servers are up -- $total_alive_count of $total_count currently running .\n\nThe complete status can be found at http://rose.cs.berkeley.edu:8080/i3_status.html";
       close MAILFILE;
       my $email_addr;

        my $email_addr;
        foreach $email_addr (@email_alert_list) {
            print "Sending email alert to $email_addr.\n";
            system ("mail -s \"I3_SERVER_STATUS: $total_alive_count of $total_count running\" $email_addr < mailfile.tmp");
        }
    }

}


if ($percent_alive < $threshold_percent) {

    my $warn_mail_last_sent_file = "warning_mail_last_sent.txt";
    open (LAST_SENT_FILE, "$warn_mail_last_sent_file");

    my $warn_mail_last_sent_time = <LAST_SENT_FILE>;

    close (LAST_SENT_FILE);
    
    my $currtime = time();
   
    if ($currtime - $warn_mail_last_sent_time > $min_interval_between_warn_mails * 60 ) {
        open (LAST_SENT_FILE, ">$warn_mail_last_sent_file");
        print LAST_SENT_FILE $currtime;
        close LAST_SENT_FILE;

        open (MAIL_FILE, ">mailfile.tmp");
        print MAIL_FILE (100 - $percent_alive)." % of the i3 servers are down.  Only $total_alive_count of $total_count are currently running.  \nAutomatic restart must have failed.  Please check manually .\n\nThe complete status can be found at http://rose.cs.berkeley.edu:8080/i3_status.html";
        close MAILFILE;

        my $email_addr;
        foreach $email_addr (@email_alert_list) {
            print "Sending email alert to $email_addr.\n";
            system ("mail -s \"WARNING: Too few i3 servers running\" $email_addr < mailfile.tmp");
        }
    }
}

# restart the i3 monitors on the hosts in RESTART_FILE

if ($attempt_restart_monitors_flag == 1 && $restart_monitors_flag > 0) {
    print "\n\nRestarting i3 server monitors on $restart_monitors_flag nodes\n";
    system ("./start_i3server_monitors.pl -h $restart_file_name");
    print "\n\nFinished i3 server monitor restart attempts.\n";
}

close CONF_DIR;

print "\nWriting HTML and TXT files\n";

# spew out details to html document
print HTML "<html>\n<head><title>i3 status</title>\n</head>\n";
print HTML "<body  bgcolor=\"#ffffff\" text=\"#000000\" link=\"#0000cc\" vlink=\"#cc00cc\" alink=\"#cc0000\"><center>\n";
print HTML "<h1><i>i3</i> status on PlanetLab</h1>\n";
print HTML "Last updated: ".localtime()." PST\n";

print HTML "<br><center><b><font color=\"#00ff00\">$total_alive_count</font></b> <i>i3</i> servers are currently running.</center><br>";

print HTML "<p>\n<center><table BORDER>\n";

print HTML "<tr BGCOLOR=\"#FFFFFF\" NOSAVE>\n<td bgcolor=\"99ccff\"><font FACE=\"Times New Roman,Times\">";
print HTML "<b>Host name</b>";
print HTML "</font></td>\n<td bgcolor=\"99ccff\"><font FACE=\"Times New Roman,Times\">Uptime";
print HTML "</font></td>\n<td bgcolor=\"99ccff\"><font FACE=\"Times New Roman,Times\">IP:Port";
print HTML "</font></td>\n<td bgcolor=\"99ccff\"><font FACE=\"Times New Roman,Times\">Chord ID";
print HTML "</font></td>\n<td bgcolor=\"99ccff\"><font FACE=\"Times New Roman,Times\">Status";
#print HTML "</font></td>\n<td><font FACE=\"Times New Roman,Times\">news_validator";
#print HTML "</font></td>\n<td><font FACE=\"Times New Roman,Times\">news_tcp_recvr";
print HTML "</font></td>\n</tr>\n";

my $host; my $bgcolor, my $str;
foreach $host (sort keys %hosts) {
    print HTML "<td><font FACE=\"Times New Roman,Times\">";
    print HTML "$ip2name{$host}";
    print TXT "$ip2name{$host} ";
    print CRON_STATUS "$ip2name{$host} ";

    print HTML "</font></td>\n<td><font FACE=\"Times New Roman,Times\">";
    if ($uptime_hash{$host} eq "") {
	print HTML "<font color=\"red\"><b><center>DEAD</center></b></font>";
    } else {
        print HTML "$uptime_hash{$host}";
    }
    
    print HTML "</font></td>\n<td><font FACE=\"Times New Roman,Times\">";
    print HTML "$host".":$port{$host}";
    print TXT "$host".":$port{$host} ";
    print CRON_STATUS "$host".":$port{$host} ";
    
    print HTML "</font></td>\n<td><font FACE=\"Times New Roman,Times\">";

    my $id_host = sprintf "%s", $id{$host};
    print HTML "$id_host";
    print TXT "$id_host ";
    
    if ($i3_count{$host} == 0) {
	$bgcolor = "red";
	$str = "Not Running";
    } elsif ($i3_count{$host} == 1) {
	$bgcolor = "orange";
	$str = "Partial";
    } elsif ($i3_count{$host} == 2) {
	$bgcolor = "\"#00ff00\"";
	$str = "Running";
    } else {
	$bgcolor = "blue";
	$str = "Invalid";
	$str = $i3_count{$host};
    }
    print HTML "</font></td>\n<td BGCOLOR=$bgcolor><font FACE=\"Times New Roman,Times\">";
    print HTML $str;
    print TXT $str." ";
    
    if (defined($lat_long{$host})) {
	print TXT $lat_long{$host}."\n";
    } elsif (defined($lat_long{$ip2name{$host}})) {
	print TXT $lat_long{$ip2name{$host}}."\n";
    } else {
	die "Latitude-Longitude missing for $host\n";
    }

    if ($cron_count{$host} > 0) {
        print CRON_STATUS "Yes";
    } else {
        print CRON_STATUS "No";
    }
   
    print CRON_STATUS "\n";

    if ($init_count{$host} == 0) {
	$bgcolor = "red";
	$str = "No";
    } elsif ($init_count{$host} == 1) {
	$bgcolor = "white";
	$str = "Running";
    } else {
	$bgcolor = "blue";
	$str = "Invalid$init_count{$host}";
    }
    # print HTML "</font></td>\n<td BGCOLOR=$bgcolor><font FACE=\"Times New Roman,Times\">";
    # print HTML $str;

    if ($tcp_count{$host} == 0) {
	$bgcolor = "red";
	$str = "No";
    } elsif ($tcp_count{$host} == 1) {
	$bgcolor = "white";
	$str = "Running";
    } else {
	$bgcolor = "blue";
	$str = "Invalid";
	$str = $tcp_count{$host};
    }
    # print HTML "</font></td>\n<td BGCOLOR=$bgcolor><font FACE=\"Times New Roman,Times\">";
    # print HTML $str;
    print HTML "</font></td>\n</tr>\n";
}
print HTML "</table></center>\n";

close HTML;
close TXT;
close CRON_STATUS;


print "update script done\n";

sub get_host_map {
    open HOSTS, "hosts_name_ip.txt";
    while (<HOSTS>) {
	chomp $_;
	my @opts = split / /, $_;
	$ip2name{$opts[1]} = $opts[0];
	$name2ip{$opts[0]} = $opts[1];
    }
    close HOSTS;
}

sub get_lat_long {
    open LATLONG, "$opt_l";
    while (<LATLONG>) {
	chomp $_,
	my @spl = split(/,/, $_);
	$lat_long{$spl[0]} = $spl[1]." ".$spl[2];
    }
    close LATLONG;
}
